[root@master build]# MON=3 OSD=3 MDS=1 MGR=1 RGW=1 NFS=1 ../src/vstart.sh -n -d
** going verbose **
rm -f core* 
hostname master
ip 192.168.188.253
port 40605
/root/ceph/build/bin/ceph-authtool --create-keyring --gen-key --name=mon. /root/ceph/build/keyring --cap mon 'allow *' 
creating /root/ceph/build/keyring
/root/ceph/build/bin/ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /root/ceph/build/keyring 
/root/ceph/build/bin/monmaptool --create --clobber --addv a [v2:192.168.188.253:40605,v1:192.168.188.253:40606] --addv b [v2:192.168.188.253:40607,v1:192.168.188.253:40608] --addv c [v2:192.168.188.253:40609,v1:192.168.188.253:40610] --print /tmp/ceph_monmap.1960193 
/root/ceph/build/bin/monmaptool: monmap file /tmp/ceph_monmap.1960193
/root/ceph/build/bin/monmaptool: generated fsid c0940106-4833-49e8-9f61-e9007cba56a1
setting min_mon_release = pacific
epoch 0
fsid c0940106-4833-49e8-9f61-e9007cba56a1
last_changed 2022-11-08T17:19:22.028763+0800
created 2022-11-08T17:19:22.028763+0800
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.188.253:40605/0,v1:192.168.188.253:40606/0] mon.a
1: [v2:192.168.188.253:40607/0,v1:192.168.188.253:40608/0] mon.b
2: [v2:192.168.188.253:40609/0,v1:192.168.188.253:40610/0] mon.c
/root/ceph/build/bin/monmaptool: writing epoch 0 to /tmp/ceph_monmap.1960193 (3 monitors)
rm -rf -- /root/ceph/build/dev/mon.a 
mkdir -p /root/ceph/build/dev/mon.a 
/root/ceph/build/bin/ceph-mon --mkfs -c /root/ceph/build/ceph.conf -i a --monmap=/tmp/ceph_monmap.1960193 --keyring=/root/ceph/build/keyring 
rm -rf -- /root/ceph/build/dev/mon.b 
mkdir -p /root/ceph/build/dev/mon.b 
/root/ceph/build/bin/ceph-mon --mkfs -c /root/ceph/build/ceph.conf -i b --monmap=/tmp/ceph_monmap.1960193 --keyring=/root/ceph/build/keyring 
rm -rf -- /root/ceph/build/dev/mon.c 
mkdir -p /root/ceph/build/dev/mon.c 
/root/ceph/build/bin/ceph-mon --mkfs -c /root/ceph/build/ceph.conf -i c --monmap=/tmp/ceph_monmap.1960193 --keyring=/root/ceph/build/keyring 
rm -- /tmp/ceph_monmap.1960193 
/root/ceph/build/bin/ceph-mon -i a -c /root/ceph/build/ceph.conf 
/root/ceph/build/bin/ceph-mon -i b -c /root/ceph/build/ceph.conf 
/root/ceph/build/bin/ceph-mon -i c -c /root/ceph/build/ceph.conf 
Populating config ...

[mgr]
        mgr/telemetry/enable = false
        mgr/telemetry/nag = false
Setting debug configs ...
creating /root/ceph/build/dev/mgr.x/keyring
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -i /root/ceph/build/dev/mgr.x/keyring auth add mgr.x mon 'allow profile mgr' mds 'allow *' osd 'allow *' 
added key for mgr.x
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring config set mgr mgr/dashboard/x/ssl_server_port 41605 --force 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring config set mgr mgr/prometheus/x/server_port 9283 --force 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring config set mgr mgr/restful/x/server_port 42605 --force 
Starting mgr.x
/root/ceph/build/bin/ceph-mgr -i x -c /root/ceph/build/ceph.conf 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
waiting for mgr dashboard module to start
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -h 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring dashboard ac-user-create admin -i /root/ceph/build/dashboard-admin-secret.txt administrator --force-password 
{"username": "admin", "password": "$2b$12$6E6jsJZfWlKrqbLt5h9CHeHSWFdgoLe9f3JKCvbXDk0zgsI0whCbO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1667899196, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": false}
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring dashboard create-self-signed-cert 
Self-signed certificate created
jiang yi 
add osd0 d7bc7f54-a2f3-4831-bc81-e8862d2ef7b3
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring osd new d7bc7f54-a2f3-4831-bc81-e8862d2ef7b3 -i /root/ceph/build/dev/osd0/new.json 
0
/root/ceph/build/bin/ceph-osd -i 0 -c /root/ceph/build/ceph.conf --mkfs --key AQA+H2pjnMMIIhAAv4O9myBq8J8ocAOd3Rjt5g== --osd-uuid d7bc7f54-a2f3-4831-bc81-e8862d2ef7b3 
2022-11-08T17:19:59.695+0800 7f9bc94645c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_bdev_label failed to open /root/ceph/build/dev/osd0/block: (2) No such file or directory
2022-11-08T17:19:59.695+0800 7f9bc94645c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_bdev_label failed to open /root/ceph/build/dev/osd0/block: (2) No such file or directory
2022-11-08T17:19:59.695+0800 7f9bc94645c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_bdev_label failed to open /root/ceph/build/dev/osd0/block: (2) No such file or directory
2022-11-08T17:19:59.697+0800 7f9bc94645c0 -1 bluestore(/root/ceph/build/dev/osd0) _read_fsid unparsable uuid 
start osd.0
osd 0 /root/ceph/build/bin/ceph-osd -i 0 -c /root/ceph/build/ceph.conf
/root/ceph/build/bin/ceph-osd -i 0 -c /root/ceph/build/ceph.conf 
jiang yi 
add osd1 f06074ad-d0aa-4d58-a2cf-b6707b8ded92
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring osd new f06074ad-d0aa-4d58-a2cf-b6707b8ded92 -i /root/ceph/build/dev/osd1/new.json 
2022-11-08T17:20:05.814+0800 7f035306e5c0 -1 Falling back to public interface
1
/root/ceph/build/bin/ceph-osd -i 1 -c /root/ceph/build/ceph.conf --mkfs --key AQBFH2pj8SG/AhAA+iwXZujRRkyTm7mgLpe69A== --osd-uuid f06074ad-d0aa-4d58-a2cf-b6707b8ded92 
2022-11-08T17:20:06.198+0800 7ff9f97565c0 -1 bluestore(/root/ceph/build/dev/osd1/block) _read_bdev_label failed to open /root/ceph/build/dev/osd1/block: (2) No such file or directory
2022-11-08T17:20:06.198+0800 7ff9f97565c0 -1 bluestore(/root/ceph/build/dev/osd1/block) _read_bdev_label failed to open /root/ceph/build/dev/osd1/block: (2) No such file or directory
2022-11-08T17:20:06.198+0800 7ff9f97565c0 -1 bluestore(/root/ceph/build/dev/osd1/block) _read_bdev_label failed to open /root/ceph/build/dev/osd1/block: (2) No such file or directory
2022-11-08T17:20:06.199+0800 7ff9f97565c0 -1 bluestore(/root/ceph/build/dev/osd1) _read_fsid unparsable uuid 
2022-11-08T17:20:09.339+0800 7f035306e5c0 -1 osd.0 0 log_to_monitors true
start osd.1
osd 1 /root/ceph/build/bin/ceph-osd -i 1 -c /root/ceph/build/ceph.conf
/root/ceph/build/bin/ceph-osd -i 1 -c /root/ceph/build/ceph.conf 
jiang yi 
add osd2 3a04601e-aff8-4c96-8c5c-04c45e5bcd59
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring osd new 3a04601e-aff8-4c96-8c5c-04c45e5bcd59 -i /root/ceph/build/dev/osd2/new.json 
2022-11-08T17:20:13.103+0800 7fa3593f35c0 -1 Falling back to public interface
2
/root/ceph/build/bin/ceph-osd -i 2 -c /root/ceph/build/ceph.conf --mkfs --key AQBMH2pjNkofERAACY7lf596fW1M/c9GP640Jw== --osd-uuid 3a04601e-aff8-4c96-8c5c-04c45e5bcd59 
2022-11-08T17:20:13.420+0800 7f46398945c0 -1 bluestore(/root/ceph/build/dev/osd2/block) _read_bdev_label failed to open /root/ceph/build/dev/osd2/block: (2) No such file or directory
2022-11-08T17:20:13.420+0800 7f46398945c0 -1 bluestore(/root/ceph/build/dev/osd2/block) _read_bdev_label failed to open /root/ceph/build/dev/osd2/block: (2) No such file or directory
2022-11-08T17:20:13.420+0800 7f46398945c0 -1 bluestore(/root/ceph/build/dev/osd2/block) _read_bdev_label failed to open /root/ceph/build/dev/osd2/block: (2) No such file or directory
2022-11-08T17:20:13.421+0800 7f46398945c0 -1 bluestore(/root/ceph/build/dev/osd2) _read_fsid unparsable uuid 
2022-11-08T17:20:16.504+0800 7fa3593f35c0 -1 osd.1 0 log_to_monitors true
start osd.2
osd 2 /root/ceph/build/bin/ceph-osd -i 2 -c /root/ceph/build/ceph.conf
/root/ceph/build/bin/ceph-osd -i 2 -c /root/ceph/build/ceph.conf 
2022-11-08T17:20:19.815+0800 7f66317b75c0 -1 Falling back to public interface
2022-11-08T17:20:23.236+0800 7f66317b75c0 -1 osd.2 0 log_to_monitors true
OSDs started
mkdir -p /root/ceph/build/dev/mds.a 
/root/ceph/build/bin/ceph-authtool --create-keyring --gen-key --name=mds.a /root/ceph/build/dev/mds.a/keyring 
creating /root/ceph/build/dev/mds.a/keyring
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring -i /root/ceph/build/dev/mds.a/keyring auth add mds.a mon 'allow profile mds' osd 'allow rw tag cephfs *=*' mds allow mgr 'allow profile mds' 
added key for mds.a
/root/ceph/build/bin/ceph-mds -i a -c /root/ceph/build/ceph.conf 
starting mds.a at 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring fs volume ls 
[]
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring fs volume create a 
Volume created successfully (no MDS daemons created)
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring fs authorize a client.fs_a / rwp 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring fs authorize * client.fs / rwp 
setting up user testid
setting up s3-test users
setting up user tester

S3 User Info:
  access key:  0555b35654ad1656d804
  secret key:  h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==

Swift User Info:
  account   : test
  user      : tester
  password  : testing

/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring auth get-or-create client.rgw.8000 mon 'allow rw' osd 'allow rwx' mgr 'allow rw' 
start rgw on http://localhost:8000
/root/ceph/build/bin/radosgw -c /root/ceph/build/ceph.conf --log-file=/root/ceph/build/out/radosgw.8000.log --admin-socket=/root/ceph/build/out/radosgw.8000.asok --pid-file=/root/ceph/build/out/radosgw.8000.pid --rgw_luarocks_location=/root/ceph/build/out/luarocks --debug-rgw=20 --debug-ms=1 -n client.rgw.8000 '--rgw_frontends=beast port=8000' 
ceph_adm auth get-or-create client.vstart mon 'allow r' osd 'allow rw pool=.nfs namespace=vstart, allow rw tag cephfs data=a' mds 'allow rw path=/' 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring auth get-or-create client.vstart mon 'allow r' osd 'allow rw pool=.nfs namespace=vstart, allow rw tag cephfs data=a' mds 'allow rw path=/' 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring mgr module enable test_orchestrator 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring orch set backend test_orchestrator 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring test_orchestrator load_data -i /root/ceph/src/pybind/mgr/test_orchestrator/dummy_data.json 
ceph_adm nfs cluster create vstart 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring nfs cluster create vstart 
vstart cluster already exists
ceph_adm nfs export create cephfs --fsname a --cluster-id vstart --pseudo-path /cephfs 
/root/ceph/build/bin/ceph -c /root/ceph/build/ceph.conf -k /root/ceph/build/keyring nfs export create cephfs --fsname a --cluster-id vstart --pseudo-path /cephfs 
{
    "bind": "/cephfs",
    "fs": "a",
    "path": "/",
    "cluster": "vstart",
    "mode": "RW"
}
rm -rf /root/ceph/build/dev/ganesha.a 
mkdir -p /root/ceph/build/dev/ganesha.a 
env CEPH_CONF=/root/ceph/build/ceph.conf ganesha-rados-grace --userid vstart -p .nfs -n vstart add a 
env CEPH_CONF=/root/ceph/build/ceph.conf ganesha-rados-grace --userid vstart -p .nfs -n vstart 
cur=1 rec=0
======================================================
a        E
env CEPH_CONF=/root/ceph/build/ceph.conf ganesha.nfsd -L /root/ceph/build/out/ganesha-a.log -f /root/ceph/build/dev/ganesha.a/ganesha-a.conf -p /root/ceph/build/out/ganesha-a.pid -N NIV_DEBUG 
env CEPH_CONF=/root/ceph/build/ceph.conf ganesha-rados-grace --userid vstart -p .nfs -n vstart 
cur=1 rec=0
======================================================
a        E
vstart ganesha daemon a started on port: 44605
Mount using: mount -t nfs -o port=<ganesha-port-num> 192.168.188.253:/cephfs mountpoint

vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output.

dashboard urls: https://192.168.188.253:41605
  w/ user/pass: admin / admin

export PYTHONPATH=/root/ceph/src/pybind:/root/ceph/build/lib/cython_modules/lib.3:/root/ceph/src/python-common:$PYTHONPATH
export LD_LIBRARY_PATH=/root/ceph/build/lib:$LD_LIBRARY_PATH
export PATH=/root/ceph/build/bin:$PATH
alias cephfs-shell=/root/ceph/src/tools/cephfs/cephfs-shell
CEPH_DEV=1

